﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Fetches.Fetch.Dto;
using Fetches.Fetch.Fetcher.AntiSpider;
using Fetches.Fetch.Fetcher.LinkExtractor;
using Fetches.Fetch.PagePool;
using Fetches.Fetch.Proxy;
using Fetches.Fetch.Scheduler;
using Fetches.Fetch.Utils.FileHosting;
using Serilog;

namespace Fetches.Fetch.Fetcher
{
    /// <summary>
    ///     Fetcher是数据的获取员，拥有浏览器、Cookie、代理设置等.
    ///     会自行根据设置从Scheduler中pull request并处理.
    ///     Fetcher不和PlatformComponents以外的对象通信,不知道是否由其他Fetcher协同处理Order,挂掉了就重新招个Fetcher
    /// </summary>
    public abstract class AbstractFetcher : IDisposable
    {
        #region 基础组件

        /// <summary>
        ///     新链接发现,和ProxyPool一个层级,由Fetcher根据Order自行初始化
        /// </summary>
        protected ILinkExtractor LinkExtractor;

        /// <summary>
        ///     需要处理的工单
        /// </summary>
        protected FetchOrder Order;

        protected AbstractScheduler Scheduler;

        protected AbstractPagePool PagePool;

        protected IProxyPool ProxyPool;

        /// <summary>
        /// 针对反爬情况的处理应对
        /// </summary>
        protected Dictionary<string, Action<AntiSpiderEvent>> AntiReactions
            = new Dictionary<string, Action<AntiSpiderEvent>>();

        private AntiSpiderDetector _antiDetector;
        #endregion

        #region 变量

        /// <summary>
        ///     FetcherId全局唯一
        /// </summary>
        public long Id { get; private set; }

        protected long TaskId { get; set; }

        public FetcherStatus Status { get; protected set; } = FetcherStatus.Preparing;

        #endregion

        /// <summary>
        /// 因为Fetcher要反射实例化所以还是无参比较方便，由Factory主动调用初始化方法
        ///     Scheduler和PagePool是Fetcher单机和分布式的关键;初始化后向Scheduler通报
        /// </summary>
        /// <param name="fetcherId">需要启动的Fetcher的ID，全局唯一</param>
        /// <param name="taskId">对于订单的每次执行有唯一ID</param>
        /// <param name="order">订单</param>
        /// <param name="scheduler">任务调度</param>
        /// <param name="pagePool">页面存储</param>
        /// <param name="proxyPool">代理池</param>
        /// <param name="linkExtractor">页面链接抽取</param>
        /// <param name="antiSpiderDetector">反爬检查，应对手段由Fetcher来负责</param>
        public async Task Initialize(long fetcherId, long taskId, FetchOrder order, AbstractScheduler scheduler,
             AbstractPagePool pagePool, IProxyPool proxyPool, ILinkExtractor linkExtractor,AntiSpiderDetector antiSpiderDetector)
        {
            Id = fetcherId;
            TaskId = taskId;
            Order = order;
            Scheduler = scheduler;
            PagePool = pagePool;
            ProxyPool = proxyPool;
            LinkExtractor = linkExtractor;
            _antiDetector = antiSpiderDetector;

            await Task.Run(() =>
            {
                if (Scheduler.IsFirstRun())
                {
                    foreach (var seedUrl in order.SeedUrls) Scheduler.Push(new Request(seedUrl) { IndexPage = true });
                    BeforeTask();
                }

                CustomInitialize();
                Status = FetcherStatus.Ready;
            });

        }

        /// <summary>
        ///     想在爬取前做一些额外的操作
        ///     由于Order需要序列化，所以不能放在Order里面
        ///     OrderSubmit由于要等价替换为Factory的Api所以也不能放
        ///     由Supervisor来通知又感觉为时已晚,所以放在Fetcher初始化的时候,Fetcher判断自己的Id=1就执行操作，其他的跳过
        /// </summary>
        public virtual void BeforeTask()
        {
        }

        /// <summary>
        ///     这个初始化时每个Fetcher执行一次
        ///     可以用来通知某Fetcher已启动、也可以用来添加自定义的反爬应对策略，别滥用
        /// </summary>
        public virtual void CustomInitialize()
        {
        }

        protected abstract ResponsePage Fetch(Request request);

        /// <summary>
        /// 获得资源文件数据
        /// </summary>
        /// <param name="refer">防盗链需要refer</param>
        /// <param name="resourceUri">资源文件地址</param>
        /// <returns></returns>
        protected abstract Task<byte[]> GetResourceAsync(string refer, Uri resourceUri);

        #region work

        /// <summary>
        ///     单个动作
        /// </summary>
        protected virtual void SingleWorkAction()
        {
            var request = Scheduler.Poll();
            if (request == null)
            {
                //没有新请求的情况下进入任务结束判断流程,判断Idle时为了防止反复报告空闲
                if (Status != FetcherStatus.Idle)
                {
                    Status = FetcherStatus.Idle;
                    Scheduler.FetcherStatusReport(this);
                }

                if (Scheduler.IsFetchFinished())
                {
                    Status = FetcherStatus.Finished;
                }
                else
                {
                    Thread.Sleep(Order.IdleSleepTime);
                }

                return;
            }

            if (Status == FetcherStatus.Idle)
            {
                //已汇报过空闲的情况下才需要重新汇报忙碌
                Status = FetcherStatus.Working;
                Scheduler.FetcherStatusReport(this);
            }

            ResponsePage responsePage = null;
            while (request.TryTimes < Order.RetryTimes && request.AntiTryTimes < Order.AntiRetryTimes)
            {
                responsePage = Fetch(request);
                responsePage.Request = request;
                var antiEvent = _antiDetector.AntiDetect(responsePage);
                if (antiEvent != null)
                {
                    //遇到反爬手段试几次，绕不过就放弃吧
                    ProcessAntiEvent(antiEvent);
                    request.AntiTryTimes++;
                    continue;
                }
                request.TryTimes++;
                Log.Debug("StatusCode={status} when fetcher {id} fetch {url}", responsePage.StatusCode, Id, request.Url.OriginalString);
                if (!Order.AcceptStatCode.Contains(responsePage.StatusCode))
                {
                    Thread.Sleep(Order.RetrySleepTime);
                    continue;
                }

                break;
            }

            Scheduler.Ack(request, responsePage != null && Order.AcceptStatCode.Contains(responsePage.StatusCode));
            if (responsePage != null)
            {
                if (Order.SpawnUrl)
                {
                    var newLinks = LinkExtractor.ExtractLinks(responsePage);
                    SpawnRequests(newLinks);
                }
                if (Order.HostingTypeList.Any())
                {
                    //先暂时改为同步的吧
                    responsePage.LocalizedSourceCode = Order.HostingRelated2Fetcher ?
                        FileHost.Instance.LocalizeResource(Order.HostingTypeList, responsePage, GetResourceAsync).Result :
                        FileHost.Instance.LocalizeResource(Order.HostingTypeList, responsePage).Result;
                }
                PagePool.Cache(responsePage);
            }
        }



        private void SpawnRequests(IList<Request> foundRequests)
        {
            Scheduler.AskForRequestsAssignment(this, foundRequests.Where(r => r.RelatedTo > 0).ToList());

            foreach (var foundRequest in foundRequests.Where(r => r.RelatedTo < 0))
                Scheduler.Push(foundRequest);
        }

        private void ProcessAntiEvent(AntiSpiderEvent antiEvent)
        {
            if (AntiReactions.TryGetValue(antiEvent.EventName, out var action))
            {
                action(antiEvent);
            }
            else
            {
                Log.Warning("Fetcher{id}未找到{event}反爬的应对手段，Fetcher自闭3分钟", Id, antiEvent.EventName);
                Thread.Sleep(3 * 60 * 1000);
            }
        }

        /// <summary>
        ///     所有的爬取逻辑都在Fetcher里面而且可定制，灵活度爆表了
        /// </summary>
        public virtual void WorkUntilSignal()
        {
            new Thread(() =>
            {
                if (Status == FetcherStatus.Working || Status == FetcherStatus.Idle)
                {
                    //防止重复通知启动
                    return;
                }
                Status = FetcherStatus.Working;
                Scheduler.FetcherStatusReport(this);
                Log.Debug("fetcher {id} is working", Id);
                while (Status != FetcherStatus.Finished && Status != FetcherStatus.Stop)
                {
                    PagePool.ReadOnly = false;
                    SingleWorkAction();
                    Thread.Sleep(Order.FetchInterval);
                }

                Dispose(); ;
                Log.Debug("fetcher {id} is stoped", Id);
            }).Start();
        }

        public void Stop()
        {
            if (Status != FetcherStatus.Finished)
            {
                Status = FetcherStatus.Stop;
            }
        }

        public virtual void Dispose()
        {
        }

        #endregion
    }
}